Exclude `deflate` from Accept-Encoding by default
Some web servers return a raw deflate stream instead of a zlib-wrapped
deflate stream defined in RFCs (called "DEFLATE") when they say
`Content-Encoding: deflate`. However FaradayMiddleware::Gzip currently
only supports the latter, so we need to keep it from telling a server
that it accepts `deflate`.
See #1018.
Akinori MUSHA
2 changed files with 35 additions and 0 deletions
+ 5
- 0
app/concerns/web_request_concern.rb
@@ -123,6 +123,11 @@ module WebRequestConcern |
||
| 123 | 123 |
|
| 124 | 124 |
builder.use FaradayMiddleware::Gzip |
| 125 | 125 |
|
| 126 |
+ unless builder.headers.any? { |key,| /\Aaccept[-_]encoding\z/i =~ key }
|
|
| 127 |
+ # Exclude `deflate` by default. See #1018. |
|
| 128 |
+ builder.headers[:accept_encoding] = 'gzip,identity' |
|
| 129 |
+ end |
|
| 130 |
+ |
|
| 126 | 131 |
case backend = faraday_backend |
| 127 | 132 |
when :typhoeus |
| 128 | 133 |
require 'typhoeus/adapters/faraday' |
+ 30
- 0
spec/models/agents/website_agent_spec.rb
@@ -227,6 +227,36 @@ describe Agents::WebsiteAgent do |
||
| 227 | 227 |
event = Event.last |
| 228 | 228 |
expect(event.payload['version']).to eq(2) |
| 229 | 229 |
end |
| 230 |
+ |
|
| 231 |
+ it 'should either avoid or support a raw deflate stream (#1018)' do |
|
| 232 |
+ stub_request(:any, /deflate/).with(headers: { 'Accept-Encoding' => /\A(?!.*deflate)/ }).
|
|
| 233 |
+ to_return(body: 'hello', |
|
| 234 |
+ status: 200) |
|
| 235 |
+ stub_request(:any, /deflate/).with(headers: { 'Accept-Encoding' => /deflate/ }).
|
|
| 236 |
+ to_return(body: '\xcb\x48\xcd\xc9\xc9\x07\x00\x06\x2c'.force_encoding(Encoding::ASCII_8BIT), |
|
| 237 |
+ headers: { 'Content-Encoding' => 'deflate' },
|
|
| 238 |
+ status: 200) |
|
| 239 |
+ |
|
| 240 |
+ site = {
|
|
| 241 |
+ 'name' => 'Some Response', |
|
| 242 |
+ 'expected_update_period_in_days' => '2', |
|
| 243 |
+ 'type' => 'text', |
|
| 244 |
+ 'url' => 'http://deflate', |
|
| 245 |
+ 'mode' => 'on_change', |
|
| 246 |
+ 'extract' => {
|
|
| 247 |
+ 'content' => { 'regexp' => '.+', 'index' => 0 }
|
|
| 248 |
+ } |
|
| 249 |
+ } |
|
| 250 |
+ checker = Agents::WebsiteAgent.new(name: "Deflate Test", options: site) |
|
| 251 |
+ checker.user = users(:bob) |
|
| 252 |
+ checker.save! |
|
| 253 |
+ |
|
| 254 |
+ expect {
|
|
| 255 |
+ checker.check |
|
| 256 |
+ }.to change { Event.count }.by(1)
|
|
| 257 |
+ event = Event.last |
|
| 258 |
+ expect(event.payload['content']).to eq('hello')
|
|
| 259 |
+ end |
|
| 230 | 260 |
end |
| 231 | 261 |
|
| 232 | 262 |
describe 'encoding' do |